"""
this file is user to download data from the Internet such as http://www.nhc.gov.cn/xcs/yqtb/list_gzbd.shtml
"""
import os
from selenium import webdriver as wbd
import time
from bs4 import BeautifulSoup as bs  # beautiful soup
driver = wbd.Firefox()
list_url = ["http://www.nhc.gov.cn/xcs/yqtb/list_gzbd.shtml",
            "http://www.nhc.gov.cn/xcs/yqtb/list_gzbd_2.shtml",
            "http://www.nhc.gov.cn/xcs/yqtb/list_gzbd_3.shtml"]
url_base = "http://www.nhc.gov.cn"
list_url_daily = []
for url_i in list_url:
    url_per_url_i = []
    driver.get(url_i)
    time.sleep(3)
    soup = bs(driver.page_source)
    soup_ul = soup.ul
    list_ul = soup_ul.findAll('a')
    for item in list_ul:
        url_per_url_i.append(url_base + item.attrs['href'])
    list_url_daily = list_url_daily + url_per_url_i
    # driver.close()
print(list_url_daily)

for url_one_day in list_url_daily:
    driver.get(url_one_day)
    time.sleep(1)
    soup = bs(driver.page_source)
    title_content = soup.find('div', class_='tit').get_text()
    time_fr = soup.find('div', class_='w1024 mb50').find('div', class_='list').find('div', class_='source').find('span', recursive=False)
    time_fr = time_fr.text
    content_daily = soup.p.text
    try:
        with open(os.path.join('../data/', title_content + '.txt'), 'w') as file_daily:
            file_daily.write(time_fr + '\n')
            print(content_daily)
            file_daily.write(content_daily.strip('\u2002'))
            file_daily.close()
    except:
        pass
